{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:50:06.511981Z",
     "start_time": "2018-09-28T13:50:06.501899Z"
    }
   },
   "outputs": [],
   "source": [
    "# see http://icsdweb.aegean.gr/awid for more info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:50:06.921121Z",
     "start_time": "2018-09-28T13:50:06.517350Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:50:07.118234Z",
     "start_time": "2018-09-28T13:50:06.926090Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# http://icsdweb.aegean.gr/awid/features.html\n",
    "\n",
    "# get the names of the features\n",
    "    \n",
    "features = ['frame.interface_id',\n",
    " 'frame.dlt',\n",
    " 'frame.offset_shift',\n",
    " 'frame.time_epoch',\n",
    " 'frame.time_delta',\n",
    " 'frame.time_delta_displayed',\n",
    " 'frame.time_relative',\n",
    " 'frame.len',\n",
    " 'frame.cap_len',\n",
    " 'frame.marked',\n",
    " 'frame.ignored',\n",
    " 'radiotap.version',\n",
    " 'radiotap.pad',\n",
    " 'radiotap.length',\n",
    " 'radiotap.present.tsft',\n",
    " 'radiotap.present.flags',\n",
    " 'radiotap.present.rate',\n",
    " 'radiotap.present.channel',\n",
    " 'radiotap.present.fhss',\n",
    " 'radiotap.present.dbm_antsignal',\n",
    " 'radiotap.present.dbm_antnoise',\n",
    " 'radiotap.present.lock_quality',\n",
    " 'radiotap.present.tx_attenuation',\n",
    " 'radiotap.present.db_tx_attenuation',\n",
    " 'radiotap.present.dbm_tx_power',\n",
    " 'radiotap.present.antenna',\n",
    " 'radiotap.present.db_antsignal',\n",
    " 'radiotap.present.db_antnoise',\n",
    " 'radiotap.present.rxflags',\n",
    " 'radiotap.present.xchannel',\n",
    " 'radiotap.present.mcs',\n",
    " 'radiotap.present.ampdu',\n",
    " 'radiotap.present.vht',\n",
    " 'radiotap.present.reserved',\n",
    " 'radiotap.present.rtap_ns',\n",
    " 'radiotap.present.vendor_ns',\n",
    " 'radiotap.present.ext',\n",
    " 'radiotap.mactime',\n",
    " 'radiotap.flags.cfp',\n",
    " 'radiotap.flags.preamble',\n",
    " 'radiotap.flags.wep',\n",
    " 'radiotap.flags.frag',\n",
    " 'radiotap.flags.fcs',\n",
    " 'radiotap.flags.datapad',\n",
    " 'radiotap.flags.badfcs',\n",
    " 'radiotap.flags.shortgi',\n",
    " 'radiotap.datarate',\n",
    " 'radiotap.channel.freq',\n",
    " 'radiotap.channel.type.turbo',\n",
    " 'radiotap.channel.type.cck',\n",
    " 'radiotap.channel.type.ofdm',\n",
    " 'radiotap.channel.type.2ghz',\n",
    " 'radiotap.channel.type.5ghz',\n",
    " 'radiotap.channel.type.passive',\n",
    " 'radiotap.channel.type.dynamic',\n",
    " 'radiotap.channel.type.gfsk',\n",
    " 'radiotap.channel.type.gsm',\n",
    " 'radiotap.channel.type.sturbo',\n",
    " 'radiotap.channel.type.half',\n",
    " 'radiotap.channel.type.quarter',\n",
    " 'radiotap.dbm_antsignal',\n",
    " 'radiotap.antenna',\n",
    " 'radiotap.rxflags.badplcp',\n",
    " 'wlan.fc.type_subtype',\n",
    " 'wlan.fc.version',\n",
    " 'wlan.fc.type',\n",
    " 'wlan.fc.subtype',\n",
    " 'wlan.fc.ds',\n",
    " 'wlan.fc.frag',\n",
    " 'wlan.fc.retry',\n",
    " 'wlan.fc.pwrmgt',\n",
    " 'wlan.fc.moredata',\n",
    " 'wlan.fc.protected',\n",
    " 'wlan.fc.order',\n",
    " 'wlan.duration',\n",
    " 'wlan.ra',\n",
    " 'wlan.da',\n",
    " 'wlan.ta',\n",
    " 'wlan.sa',\n",
    " 'wlan.bssid',\n",
    " 'wlan.frag',\n",
    " 'wlan.seq',\n",
    " 'wlan.bar.type',\n",
    " 'wlan.ba.control.ackpolicy',\n",
    " 'wlan.ba.control.multitid',\n",
    " 'wlan.ba.control.cbitmap',\n",
    " 'wlan.bar.compressed.tidinfo',\n",
    " 'wlan.ba.bm',\n",
    " 'wlan.fcs_good',\n",
    " 'wlan_mgt.fixed.capabilities.ess',\n",
    " 'wlan_mgt.fixed.capabilities.ibss',\n",
    " 'wlan_mgt.fixed.capabilities.cfpoll.ap',\n",
    " 'wlan_mgt.fixed.capabilities.privacy',\n",
    " 'wlan_mgt.fixed.capabilities.preamble',\n",
    " 'wlan_mgt.fixed.capabilities.pbcc',\n",
    " 'wlan_mgt.fixed.capabilities.agility',\n",
    " 'wlan_mgt.fixed.capabilities.spec_man',\n",
    " 'wlan_mgt.fixed.capabilities.short_slot_time',\n",
    " 'wlan_mgt.fixed.capabilities.apsd',\n",
    " 'wlan_mgt.fixed.capabilities.radio_measurement',\n",
    " 'wlan_mgt.fixed.capabilities.dsss_ofdm',\n",
    " 'wlan_mgt.fixed.capabilities.del_blk_ack',\n",
    " 'wlan_mgt.fixed.capabilities.imm_blk_ack',\n",
    " 'wlan_mgt.fixed.listen_ival',\n",
    " 'wlan_mgt.fixed.current_ap',\n",
    " 'wlan_mgt.fixed.status_code',\n",
    " 'wlan_mgt.fixed.timestamp',\n",
    " 'wlan_mgt.fixed.beacon',\n",
    " 'wlan_mgt.fixed.aid',\n",
    " 'wlan_mgt.fixed.reason_code',\n",
    " 'wlan_mgt.fixed.auth.alg',\n",
    " 'wlan_mgt.fixed.auth_seq',\n",
    " 'wlan_mgt.fixed.category_code',\n",
    " 'wlan_mgt.fixed.htact',\n",
    " 'wlan_mgt.fixed.chanwidth',\n",
    " 'wlan_mgt.fixed.fragment',\n",
    " 'wlan_mgt.fixed.sequence',\n",
    " 'wlan_mgt.tagged.all',\n",
    " 'wlan_mgt.ssid',\n",
    " 'wlan_mgt.ds.current_channel',\n",
    " 'wlan_mgt.tim.dtim_count',\n",
    " 'wlan_mgt.tim.dtim_period',\n",
    " 'wlan_mgt.tim.bmapctl.multicast',\n",
    " 'wlan_mgt.tim.bmapctl.offset',\n",
    " 'wlan_mgt.country_info.environment',\n",
    " 'wlan_mgt.rsn.version',\n",
    " 'wlan_mgt.rsn.gcs.type',\n",
    " 'wlan_mgt.rsn.pcs.count',\n",
    " 'wlan_mgt.rsn.akms.count',\n",
    " 'wlan_mgt.rsn.akms.type',\n",
    " 'wlan_mgt.rsn.capabilities.preauth',\n",
    " 'wlan_mgt.rsn.capabilities.no_pairwise',\n",
    " 'wlan_mgt.rsn.capabilities.ptksa_replay_counter',\n",
    " 'wlan_mgt.rsn.capabilities.gtksa_replay_counter',\n",
    " 'wlan_mgt.rsn.capabilities.mfpr',\n",
    " 'wlan_mgt.rsn.capabilities.mfpc',\n",
    " 'wlan_mgt.rsn.capabilities.peerkey',\n",
    " 'wlan_mgt.tcprep.trsmt_pow',\n",
    " 'wlan_mgt.tcprep.link_mrg',\n",
    " 'wlan.wep.iv',\n",
    " 'wlan.wep.key',\n",
    " 'wlan.wep.icv',\n",
    " 'wlan.tkip.extiv',\n",
    " 'wlan.ccmp.extiv',\n",
    " 'wlan.qos.tid',\n",
    " 'wlan.qos.priority',\n",
    " 'wlan.qos.eosp',\n",
    " 'wlan.qos.ack',\n",
    " 'wlan.qos.amsdupresent',\n",
    " 'wlan.qos.buf_state_indicated',\n",
    " 'wlan.qos.bit4',\n",
    " 'wlan.qos.txop_dur_req',\n",
    " 'wlan.qos.buf_state_indicated',\n",
    " 'data.len',\n",
    " 'class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:51:30.711842Z",
     "start_time": "2018-09-28T13:50:52.914417Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sinanozdemir/Desktop/cyber/env/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2714: DtypeWarning: Columns (37,38,39,40,41,42,43,44,45,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,74,88) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1795575, 155)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import a training set\n",
    "awid = pd.read_csv(\"../data/AWID-CLS-R-Trn.csv\", header=None, names=features)\n",
    "\n",
    "# see the number of rows/columns\n",
    "awid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:51:30.772365Z",
     "start_time": "2018-09-28T13:51:30.717247Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frame.interface_id</th>\n",
       "      <th>frame.dlt</th>\n",
       "      <th>frame.offset_shift</th>\n",
       "      <th>frame.time_epoch</th>\n",
       "      <th>frame.time_delta</th>\n",
       "      <th>frame.time_delta_displayed</th>\n",
       "      <th>frame.time_relative</th>\n",
       "      <th>frame.len</th>\n",
       "      <th>frame.cap_len</th>\n",
       "      <th>frame.marked</th>\n",
       "      <th>...</th>\n",
       "      <th>wlan.qos.priority</th>\n",
       "      <th>wlan.qos.eosp</th>\n",
       "      <th>wlan.qos.ack</th>\n",
       "      <th>wlan.qos.amsdupresent</th>\n",
       "      <th>wlan.qos.buf_state_indicated</th>\n",
       "      <th>wlan.qos.bit4</th>\n",
       "      <th>wlan.qos.txop_dur_req</th>\n",
       "      <th>wlan.qos.buf_state_indicated.1</th>\n",
       "      <th>data.len</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>261</td>\n",
       "      <td>261</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.024271</td>\n",
       "      <td>0.024271</td>\n",
       "      <td>0.024271</td>\n",
       "      <td>185</td>\n",
       "      <td>185</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.001631</td>\n",
       "      <td>0.001631</td>\n",
       "      <td>0.025902</td>\n",
       "      <td>185</td>\n",
       "      <td>185</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.055325</td>\n",
       "      <td>0.055325</td>\n",
       "      <td>0.081227</td>\n",
       "      <td>159</td>\n",
       "      <td>159</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.000415</td>\n",
       "      <td>0.000415</td>\n",
       "      <td>0.081642</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 155 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   frame.interface_id frame.dlt  frame.offset_shift  frame.time_epoch  \\\n",
       "0                   0         ?                 0.0      1.393661e+09   \n",
       "1                   0         ?                 0.0      1.393661e+09   \n",
       "2                   0         ?                 0.0      1.393661e+09   \n",
       "3                   0         ?                 0.0      1.393661e+09   \n",
       "4                   0         ?                 0.0      1.393661e+09   \n",
       "\n",
       "   frame.time_delta  frame.time_delta_displayed  frame.time_relative  \\\n",
       "0          0.000000                    0.000000             0.000000   \n",
       "1          0.024271                    0.024271             0.024271   \n",
       "2          0.001631                    0.001631             0.025902   \n",
       "3          0.055325                    0.055325             0.081227   \n",
       "4          0.000415                    0.000415             0.081642   \n",
       "\n",
       "   frame.len  frame.cap_len  frame.marked   ...    wlan.qos.priority  \\\n",
       "0        261            261             0   ...                    ?   \n",
       "1        185            185             0   ...                    ?   \n",
       "2        185            185             0   ...                    ?   \n",
       "3        159            159             0   ...                    ?   \n",
       "4         54             54             0   ...                    ?   \n",
       "\n",
       "   wlan.qos.eosp  wlan.qos.ack  wlan.qos.amsdupresent  \\\n",
       "0              ?             ?                      ?   \n",
       "1              ?             ?                      ?   \n",
       "2              ?             ?                      ?   \n",
       "3              ?             ?                      ?   \n",
       "4              ?             ?                      ?   \n",
       "\n",
       "   wlan.qos.buf_state_indicated  wlan.qos.bit4  wlan.qos.txop_dur_req  \\\n",
       "0                             ?              ?                      ?   \n",
       "1                             ?              ?                      ?   \n",
       "2                             ?              ?                      ?   \n",
       "3                             ?              ?                      ?   \n",
       "4                             ?              ?                      ?   \n",
       "\n",
       "   wlan.qos.buf_state_indicated.1  data.len   class  \n",
       "0                               ?         ?  normal  \n",
       "1                               ?         ?  normal  \n",
       "2                               ?         ?  normal  \n",
       "3                               ?         ?  normal  \n",
       "4                               ?         ?  normal  \n",
       "\n",
       "[5 rows x 155 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# they use ? as a null attribute. We will eventually have to replace them with None values\n",
    "awid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:51:31.028627Z",
     "start_time": "2018-09-28T13:51:30.775326Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "normal           0.909564\n",
       "injection        0.036411\n",
       "impersonation    0.027023\n",
       "flooding         0.027002\n",
       "Name: class, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see the distribution of response vars\n",
    "awid['class'].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:51:44.440815Z",
     "start_time": "2018-09-28T13:51:31.034203Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "frame.interface_id                                0\n",
       "frame.dlt                                         0\n",
       "frame.offset_shift                                0\n",
       "frame.time_epoch                                  0\n",
       "frame.time_delta                                  0\n",
       "frame.time_delta_displayed                        0\n",
       "frame.time_relative                               0\n",
       "frame.len                                         0\n",
       "frame.cap_len                                     0\n",
       "frame.marked                                      0\n",
       "frame.ignored                                     0\n",
       "radiotap.version                                  0\n",
       "radiotap.pad                                      0\n",
       "radiotap.length                                   0\n",
       "radiotap.present.tsft                             0\n",
       "radiotap.present.flags                            0\n",
       "radiotap.present.rate                             0\n",
       "radiotap.present.channel                          0\n",
       "radiotap.present.fhss                             0\n",
       "radiotap.present.dbm_antsignal                    0\n",
       "radiotap.present.dbm_antnoise                     0\n",
       "radiotap.present.lock_quality                     0\n",
       "radiotap.present.tx_attenuation                   0\n",
       "radiotap.present.db_tx_attenuation                0\n",
       "radiotap.present.dbm_tx_power                     0\n",
       "radiotap.present.antenna                          0\n",
       "radiotap.present.db_antsignal                     0\n",
       "radiotap.present.db_antnoise                      0\n",
       "radiotap.present.rxflags                          0\n",
       "radiotap.present.xchannel                         0\n",
       "                                                 ..\n",
       "wlan_mgt.rsn.version                              0\n",
       "wlan_mgt.rsn.gcs.type                             0\n",
       "wlan_mgt.rsn.pcs.count                            0\n",
       "wlan_mgt.rsn.akms.count                           0\n",
       "wlan_mgt.rsn.akms.type                            0\n",
       "wlan_mgt.rsn.capabilities.preauth                 0\n",
       "wlan_mgt.rsn.capabilities.no_pairwise             0\n",
       "wlan_mgt.rsn.capabilities.ptksa_replay_counter    0\n",
       "wlan_mgt.rsn.capabilities.gtksa_replay_counter    0\n",
       "wlan_mgt.rsn.capabilities.mfpr                    0\n",
       "wlan_mgt.rsn.capabilities.mfpc                    0\n",
       "wlan_mgt.rsn.capabilities.peerkey                 0\n",
       "wlan_mgt.tcprep.trsmt_pow                         0\n",
       "wlan_mgt.tcprep.link_mrg                          0\n",
       "wlan.wep.iv                                       0\n",
       "wlan.wep.key                                      0\n",
       "wlan.wep.icv                                      0\n",
       "wlan.tkip.extiv                                   0\n",
       "wlan.ccmp.extiv                                   0\n",
       "wlan.qos.tid                                      0\n",
       "wlan.qos.priority                                 0\n",
       "wlan.qos.eosp                                     0\n",
       "wlan.qos.ack                                      0\n",
       "wlan.qos.amsdupresent                             0\n",
       "wlan.qos.buf_state_indicated                      0\n",
       "wlan.qos.bit4                                     0\n",
       "wlan.qos.txop_dur_req                             0\n",
       "wlan.qos.buf_state_indicated.1                    0\n",
       "data.len                                          0\n",
       "class                                             0\n",
       "Length: 155, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# claims there are no null values because of the ?'s'\n",
    "awid.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:52:19.004572Z",
     "start_time": "2018-09-28T13:51:44.446396Z"
    }
   },
   "outputs": [],
   "source": [
    "# replace the ? marks with None\n",
    "awid.replace({\"?\": None}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:52:32.432462Z",
     "start_time": "2018-09-28T13:52:19.011439Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "frame.interface_id                                      0\n",
       "frame.dlt                                         1795575\n",
       "frame.offset_shift                                      0\n",
       "frame.time_epoch                                        0\n",
       "frame.time_delta                                        0\n",
       "frame.time_delta_displayed                              0\n",
       "frame.time_relative                                     0\n",
       "frame.len                                               0\n",
       "frame.cap_len                                           0\n",
       "frame.marked                                            0\n",
       "frame.ignored                                           0\n",
       "radiotap.version                                        0\n",
       "radiotap.pad                                            0\n",
       "radiotap.length                                         0\n",
       "radiotap.present.tsft                                   0\n",
       "radiotap.present.flags                                  0\n",
       "radiotap.present.rate                                   0\n",
       "radiotap.present.channel                                0\n",
       "radiotap.present.fhss                                   0\n",
       "radiotap.present.dbm_antsignal                          0\n",
       "radiotap.present.dbm_antnoise                           0\n",
       "radiotap.present.lock_quality                           0\n",
       "radiotap.present.tx_attenuation                         0\n",
       "radiotap.present.db_tx_attenuation                      0\n",
       "radiotap.present.dbm_tx_power                           0\n",
       "radiotap.present.antenna                                0\n",
       "radiotap.present.db_antsignal                           0\n",
       "radiotap.present.db_antnoise                            0\n",
       "radiotap.present.rxflags                                0\n",
       "radiotap.present.xchannel                               0\n",
       "                                                   ...   \n",
       "wlan_mgt.rsn.version                              1718631\n",
       "wlan_mgt.rsn.gcs.type                             1718631\n",
       "wlan_mgt.rsn.pcs.count                            1718631\n",
       "wlan_mgt.rsn.akms.count                           1718633\n",
       "wlan_mgt.rsn.akms.type                            1718651\n",
       "wlan_mgt.rsn.capabilities.preauth                 1718633\n",
       "wlan_mgt.rsn.capabilities.no_pairwise             1718633\n",
       "wlan_mgt.rsn.capabilities.ptksa_replay_counter    1718633\n",
       "wlan_mgt.rsn.capabilities.gtksa_replay_counter    1718633\n",
       "wlan_mgt.rsn.capabilities.mfpr                    1718633\n",
       "wlan_mgt.rsn.capabilities.mfpc                    1718633\n",
       "wlan_mgt.rsn.capabilities.peerkey                 1718633\n",
       "wlan_mgt.tcprep.trsmt_pow                         1795536\n",
       "wlan_mgt.tcprep.link_mrg                          1795536\n",
       "wlan.wep.iv                                        944820\n",
       "wlan.wep.key                                       909831\n",
       "wlan.wep.icv                                       944820\n",
       "wlan.tkip.extiv                                   1763655\n",
       "wlan.ccmp.extiv                                   1792506\n",
       "wlan.qos.tid                                      1133234\n",
       "wlan.qos.priority                                 1133234\n",
       "wlan.qos.eosp                                     1279874\n",
       "wlan.qos.ack                                      1133234\n",
       "wlan.qos.amsdupresent                             1134226\n",
       "wlan.qos.buf_state_indicated                      1795575\n",
       "wlan.qos.bit4                                     1648935\n",
       "wlan.qos.txop_dur_req                             1648935\n",
       "wlan.qos.buf_state_indicated.1                    1279874\n",
       "data.len                                           903021\n",
       "class                                                   0\n",
       "Length: 155, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Many missing pieces of data!\n",
    "awid.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:52:45.265068Z",
     "start_time": "2018-09-28T13:52:32.437788Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(72,)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# goal is to remove columns that have over 50% of missing data\n",
    "columns_with_mostly_null_data = awid.columns[awid.isnull().mean() >= 0.5]\n",
    "\n",
    "# 72 columns are going to be affected!\n",
    "columns_with_mostly_null_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:52:47.069733Z",
     "start_time": "2018-09-28T13:52:45.270778Z"
    }
   },
   "outputs": [],
   "source": [
    "# drop the columns with over half missing data\n",
    "awid.drop(columns_with_mostly_null_data, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:52:47.095626Z",
     "start_time": "2018-09-28T13:52:47.075018Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1795575, 83)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "awid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:52:57.278809Z",
     "start_time": "2018-09-28T13:52:47.101532Z"
    }
   },
   "outputs": [],
   "source": [
    "# now drop the rows that have missing values\n",
    "awid.dropna(inplace=True)  # drop rows with null data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:52:57.323887Z",
     "start_time": "2018-09-28T13:52:57.283484Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1339406, 83)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lost 456,169 rows\n",
    "awid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:52:57.565338Z",
     "start_time": "2018-09-28T13:52:57.339766Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "normal           0.878763\n",
       "injection        0.048812\n",
       "impersonation    0.036227\n",
       "flooding         0.036198\n",
       "Name: class, dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# doesn't affect our distribution too much\n",
    "\n",
    "# 0.878763  is our null accuracy. Our model must be better than this number to be a contender\n",
    "awid['class'].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:52:58.018310Z",
     "start_time": "2018-09-28T13:52:57.570171Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1339406, 45)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# only select numerical columns for our ML algorithms, there should be more..\n",
    "awid.select_dtypes(['number']).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:53:25.799430Z",
     "start_time": "2018-09-28T13:52:58.035055Z"
    }
   },
   "outputs": [],
   "source": [
    "# transform all columns into numerical dtypes\n",
    "for col in awid.columns:\n",
    "    awid[col] = pd.to_numeric(awid[col], errors='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:53:27.762882Z",
     "start_time": "2018-09-28T13:53:25.805007Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1339406, 74)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# that makes more sense\n",
    "awid.select_dtypes(['number']).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:53:34.625597Z",
     "start_time": "2018-09-28T13:53:27.768722Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frame.interface_id</th>\n",
       "      <th>frame.offset_shift</th>\n",
       "      <th>frame.time_epoch</th>\n",
       "      <th>frame.time_delta</th>\n",
       "      <th>frame.time_delta_displayed</th>\n",
       "      <th>frame.time_relative</th>\n",
       "      <th>frame.len</th>\n",
       "      <th>frame.cap_len</th>\n",
       "      <th>frame.marked</th>\n",
       "      <th>frame.ignored</th>\n",
       "      <th>...</th>\n",
       "      <th>wlan.fc.frag</th>\n",
       "      <th>wlan.fc.retry</th>\n",
       "      <th>wlan.fc.pwrmgt</th>\n",
       "      <th>wlan.fc.moredata</th>\n",
       "      <th>wlan.fc.protected</th>\n",
       "      <th>wlan.fc.order</th>\n",
       "      <th>wlan.duration</th>\n",
       "      <th>wlan.frag</th>\n",
       "      <th>wlan.seq</th>\n",
       "      <th>wlan.fcs_good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1339406.0</td>\n",
       "      <td>1339406.0</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1339406.0</td>\n",
       "      <td>1339406.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1339406.0</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1339406.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393663e+09</td>\n",
       "      <td>2.435200e-03</td>\n",
       "      <td>2.435200e-03</td>\n",
       "      <td>1.842787e+03</td>\n",
       "      <td>6.771728e+02</td>\n",
       "      <td>6.771728e+02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5.024615e-04</td>\n",
       "      <td>1.703964e-01</td>\n",
       "      <td>2.870825e-02</td>\n",
       "      <td>2.376427e-03</td>\n",
       "      <td>6.612961e-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.028715e+02</td>\n",
       "      <td>2.165885e-03</td>\n",
       "      <td>1.910773e+03</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.370012e+02</td>\n",
       "      <td>6.827733e-03</td>\n",
       "      <td>6.827733e-03</td>\n",
       "      <td>8.370012e+02</td>\n",
       "      <td>6.997003e+02</td>\n",
       "      <td>6.997003e+02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.241003e-02</td>\n",
       "      <td>3.759808e-01</td>\n",
       "      <td>1.669853e-01</td>\n",
       "      <td>4.869067e-02</td>\n",
       "      <td>4.732692e-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.237048e+02</td>\n",
       "      <td>1.176732e-01</td>\n",
       "      <td>1.200198e+03</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.400000e+01</td>\n",
       "      <td>5.400000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393662e+09</td>\n",
       "      <td>4.430000e-04</td>\n",
       "      <td>4.430000e-04</td>\n",
       "      <td>1.103988e+03</td>\n",
       "      <td>9.800000e+01</td>\n",
       "      <td>9.800000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.400000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>8.410000e+02</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393663e+09</td>\n",
       "      <td>7.790000e-04</td>\n",
       "      <td>7.790000e-04</td>\n",
       "      <td>1.894319e+03</td>\n",
       "      <td>1.590000e+02</td>\n",
       "      <td>1.590000e+02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.400000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.812000e+03</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393664e+09</td>\n",
       "      <td>1.393000e-03</td>\n",
       "      <td>1.393000e-03</td>\n",
       "      <td>2.374825e+03</td>\n",
       "      <td>1.552000e+03</td>\n",
       "      <td>1.552000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.580000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.941000e+03</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393665e+09</td>\n",
       "      <td>3.502630e-01</td>\n",
       "      <td>3.502630e-01</td>\n",
       "      <td>3.599571e+03</td>\n",
       "      <td>1.570000e+03</td>\n",
       "      <td>1.570000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.200000e+02</td>\n",
       "      <td>1.200000e+01</td>\n",
       "      <td>4.095000e+03</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 74 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       frame.interface_id  frame.offset_shift  frame.time_epoch  \\\n",
       "count           1339406.0           1339406.0      1.339406e+06   \n",
       "mean                  0.0                 0.0      1.393663e+09   \n",
       "std                   0.0                 0.0      8.370012e+02   \n",
       "min                   0.0                 0.0      1.393661e+09   \n",
       "25%                   0.0                 0.0      1.393662e+09   \n",
       "50%                   0.0                 0.0      1.393663e+09   \n",
       "75%                   0.0                 0.0      1.393664e+09   \n",
       "max                   0.0                 0.0      1.393665e+09   \n",
       "\n",
       "       frame.time_delta  frame.time_delta_displayed  frame.time_relative  \\\n",
       "count      1.339406e+06                1.339406e+06         1.339406e+06   \n",
       "mean       2.435200e-03                2.435200e-03         1.842787e+03   \n",
       "std        6.827733e-03                6.827733e-03         8.370012e+02   \n",
       "min        0.000000e+00                0.000000e+00         0.000000e+00   \n",
       "25%        4.430000e-04                4.430000e-04         1.103988e+03   \n",
       "50%        7.790000e-04                7.790000e-04         1.894319e+03   \n",
       "75%        1.393000e-03                1.393000e-03         2.374825e+03   \n",
       "max        3.502630e-01                3.502630e-01         3.599571e+03   \n",
       "\n",
       "          frame.len  frame.cap_len  frame.marked  frame.ignored  \\\n",
       "count  1.339406e+06   1.339406e+06     1339406.0      1339406.0   \n",
       "mean   6.771728e+02   6.771728e+02           0.0            0.0   \n",
       "std    6.997003e+02   6.997003e+02           0.0            0.0   \n",
       "min    5.400000e+01   5.400000e+01           0.0            0.0   \n",
       "25%    9.800000e+01   9.800000e+01           0.0            0.0   \n",
       "50%    1.590000e+02   1.590000e+02           0.0            0.0   \n",
       "75%    1.552000e+03   1.552000e+03           0.0            0.0   \n",
       "max    1.570000e+03   1.570000e+03           0.0            0.0   \n",
       "\n",
       "           ...        wlan.fc.frag  wlan.fc.retry  wlan.fc.pwrmgt  \\\n",
       "count      ...        1.339406e+06   1.339406e+06    1.339406e+06   \n",
       "mean       ...        5.024615e-04   1.703964e-01    2.870825e-02   \n",
       "std        ...        2.241003e-02   3.759808e-01    1.669853e-01   \n",
       "min        ...        0.000000e+00   0.000000e+00    0.000000e+00   \n",
       "25%        ...        0.000000e+00   0.000000e+00    0.000000e+00   \n",
       "50%        ...        0.000000e+00   0.000000e+00    0.000000e+00   \n",
       "75%        ...        0.000000e+00   0.000000e+00    0.000000e+00   \n",
       "max        ...        1.000000e+00   1.000000e+00    1.000000e+00   \n",
       "\n",
       "       wlan.fc.moredata  wlan.fc.protected  wlan.fc.order  wlan.duration  \\\n",
       "count      1.339406e+06       1.339406e+06      1339406.0   1.339406e+06   \n",
       "mean       2.376427e-03       6.612961e-01            0.0   1.028715e+02   \n",
       "std        4.869067e-02       4.732692e-01            0.0   1.237048e+02   \n",
       "min        0.000000e+00       0.000000e+00            0.0   0.000000e+00   \n",
       "25%        0.000000e+00       0.000000e+00            0.0   4.400000e+01   \n",
       "50%        0.000000e+00       1.000000e+00            0.0   4.400000e+01   \n",
       "75%        0.000000e+00       1.000000e+00            0.0   2.580000e+02   \n",
       "max        1.000000e+00       1.000000e+00            0.0   3.200000e+02   \n",
       "\n",
       "          wlan.frag      wlan.seq  wlan.fcs_good  \n",
       "count  1.339406e+06  1.339406e+06      1339406.0  \n",
       "mean   2.165885e-03  1.910773e+03            1.0  \n",
       "std    1.176732e-01  1.200198e+03            0.0  \n",
       "min    0.000000e+00  0.000000e+00            1.0  \n",
       "25%    0.000000e+00  8.410000e+02            1.0  \n",
       "50%    0.000000e+00  1.812000e+03            1.0  \n",
       "75%    0.000000e+00  2.941000e+03            1.0  \n",
       "max    1.200000e+01  4.095000e+03            1.0  \n",
       "\n",
       "[8 rows x 74 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# basic descroptive statistics\n",
    "awid.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:53:35.240767Z",
     "start_time": "2018-09-28T13:53:34.630557Z"
    }
   },
   "outputs": [],
   "source": [
    "X, y = awid.select_dtypes(['number']), awid['class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:53:42.113399Z",
     "start_time": "2018-09-28T13:53:35.246120Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB(priors=None, var_smoothing=1e-09)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# do a basic naive bayes fitting\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "nb = GaussianNB()\n",
    "\n",
    "# fit our model to the data\n",
    "nb.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:54:09.320655Z",
     "start_time": "2018-09-28T13:53:42.118984Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(389185, 83)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read in the test data and do the same transformations to it to match the training data\n",
    "awid_test = pd.read_csv(\"../data/AWID-CLS-R-Tst.csv\", header=None, names=features)\n",
    "# drop the problematic columns\n",
    "awid_test.drop(columns_with_mostly_null_data, axis=1, inplace=True)\n",
    "# replace ? with None\n",
    "awid_test.replace({\"?\": None}, inplace=True)\n",
    "# drop the rows with null data\n",
    "awid_test.dropna(inplace=True)  # drop rows with null data\n",
    "# convert columns to numerical values\n",
    "for col in awid_test.columns:\n",
    "    awid_test[col] = pd.to_numeric(awid_test[col], errors='ignore')\n",
    "awid_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:51:13.811466Z",
     "start_time": "2018-09-11T17:51:07.434162Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:54:09.350066Z",
     "start_time": "2018-09-28T13:54:09.326854Z"
    }
   },
   "outputs": [],
   "source": [
    "# basic metric, accuracy\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:54:12.080061Z",
     "start_time": "2018-09-28T13:54:09.356283Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.26535452291326744"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = awid_test.select_dtypes(['number'])\n",
    "y_test = awid_test['class']\n",
    "\n",
    "# simple function to test the accuracy of a model fitted on training data on our testing data\n",
    "def get_test_accuracy_of(model):\n",
    "    y_preds = model.predict(X_test)\n",
    "    return accuracy_score(y_preds, y_test)\n",
    "    \n",
    "# naive abyes does very poorly on its own!\n",
    "get_test_accuracy_of(nb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:54:35.770988Z",
     "start_time": "2018-09-28T13:54:12.085848Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sinanozdemir/Desktop/cyber/env/lib/python2.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/sinanozdemir/Desktop/cyber/env/lib/python2.7/site-packages/sklearn/linear_model/logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.015773989233911892"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lr = LogisticRegression()\n",
    "\n",
    "lr.fit(X, y)\n",
    "\n",
    "# Logistic Regressions does even worse\n",
    "get_test_accuracy_of(lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:54:46.227279Z",
     "start_time": "2018-09-28T13:54:35.777098Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9280830453383352"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "tree = DecisionTreeClassifier()\n",
    "\n",
    "tree.fit(X, y)\n",
    "\n",
    "# Tree does very well!\n",
    "get_test_accuracy_of(tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:54:46.989783Z",
     "start_time": "2018-09-28T13:54:46.232595Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>frame.cap_len</td>\n",
       "      <td>0.222489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>frame.time_delta_displayed</td>\n",
       "      <td>0.221133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>wlan.fc.protected</td>\n",
       "      <td>0.146001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>wlan.duration</td>\n",
       "      <td>0.127674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>frame.time_relative</td>\n",
       "      <td>0.077353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>frame.len</td>\n",
       "      <td>0.067667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>wlan.fc.type</td>\n",
       "      <td>0.039926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>wlan.seq</td>\n",
       "      <td>0.027947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>wlan.fc.retry</td>\n",
       "      <td>0.019839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>radiotap.dbm_antsignal</td>\n",
       "      <td>0.014197</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       feature  importance\n",
       "7                frame.cap_len    0.222489\n",
       "4   frame.time_delta_displayed    0.221133\n",
       "68           wlan.fc.protected    0.146001\n",
       "70               wlan.duration    0.127674\n",
       "5          frame.time_relative    0.077353\n",
       "6                    frame.len    0.067667\n",
       "62                wlan.fc.type    0.039926\n",
       "72                    wlan.seq    0.027947\n",
       "65               wlan.fc.retry    0.019839\n",
       "58      radiotap.dbm_antsignal    0.014197"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see gini scores of decision tree features\n",
    "pd.DataFrame({'feature':awid.select_dtypes(['number']).columns, \n",
    "              'importance':tree.feature_importances_}).sort_values('importance', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:55:01.220778Z",
     "start_time": "2018-09-28T13:54:46.996043Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sinanozdemir/Desktop/cyber/env/lib/python2.7/site-packages/sklearn/ensemble/forest.py:248: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9357349332579622"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest = RandomForestClassifier()\n",
    "\n",
    "forest.fit(X, y)\n",
    "\n",
    "# Random Forest does slightly worse\n",
    "get_test_accuracy_of(forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:01:06.692457Z",
     "start_time": "2018-09-11T18:01:06.681432Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:58:06.201842Z",
     "start_time": "2018-09-28T13:55:01.227317Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sinanozdemir/Desktop/cyber/env/lib/python2.7/site-packages/sklearn/model_selection/_split.py:1943: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/Users/sinanozdemir/Desktop/cyber/env/lib/python2.7/site-packages/sklearn/preprocessing/data.py:617: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/Users/sinanozdemir/Desktop/cyber/env/lib/python2.7/site-packages/sklearn/base.py:465: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, y, **fit_params).transform(X)\n",
      "/Users/sinanozdemir/Desktop/cyber/env/lib/python2.7/site-packages/sklearn/pipeline.py:451: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  Xt = transform.transform(Xt)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.926258720145946"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a pipeline that will scale the numerical data and then feed the resulting data into a decision tree\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "preprocessing = Pipeline([\n",
    "    (\"scale\", StandardScaler()),\n",
    "])\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    (\"preprocessing\", preprocessing),\n",
    "    (\"classifier\", DecisionTreeClassifier())\n",
    "])\n",
    "\n",
    "# try varying levels of depth\n",
    "params = {\n",
    "    \"classifier__max_depth\": [None, 3, 5, 10], \n",
    "         }\n",
    "\n",
    "# instantiate a gridsearch module\n",
    "grid = GridSearchCV(pipeline, params)\n",
    "# fit the module\n",
    "grid.fit(X, y)\n",
    "\n",
    "# test the best model\n",
    "get_test_accuracy_of(grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T14:01:29.147702Z",
     "start_time": "2018-09-28T13:58:06.207547Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8893431144571348"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# try the same thing with a random forest\n",
    "\n",
    "preprocessing = Pipeline([\n",
    "    (\"scale\", StandardScaler()),\n",
    "])\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    (\"preprocessing\", preprocessing),\n",
    "    (\"classifier\", RandomForestClassifier())\n",
    "])\n",
    "\n",
    "# try varying levels of depth\n",
    "params = {\n",
    "    \"classifier__max_depth\": [None, 3, 5, 10], \n",
    "         }\n",
    "\n",
    "grid = GridSearchCV(pipeline, params)\n",
    "grid.fit(X, y)\n",
    "# best accuracy so far!\n",
    "get_test_accuracy_of(grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T14:33:33.656531Z",
     "start_time": "2018-09-28T14:33:33.362231Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1339406,)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "encoder = LabelEncoder()\n",
    "encoded_y = encoder.fit_transform(y)\n",
    "encoded_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T14:33:41.340926Z",
     "start_time": "2018-09-28T14:33:41.330167Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3, 3, 3, ..., 3, 3, 3])"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T14:33:38.964439Z",
     "start_time": "2018-09-28T14:33:38.700243Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1339406, 4)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelBinarizer\n",
    "binarizer = LabelBinarizer()\n",
    "binarized_y = binarizer.fit_transform(encoded_y)\n",
    "binarized_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T14:33:46.828293Z",
     "start_time": "2018-09-28T14:33:46.817327Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 1],\n",
       "       [0, 0, 0, 1],\n",
       "       [0, 0, 0, 1],\n",
       "       [0, 0, 0, 1],\n",
       "       [0, 0, 0, 1]])"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarized_y[:5,]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T14:33:47.850255Z",
     "start_time": "2018-09-28T14:33:47.837110Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    normal\n",
       "1    normal\n",
       "2    normal\n",
       "3    normal\n",
       "4    normal\n",
       "Name: class, dtype: object"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T14:33:59.351975Z",
     "start_time": "2018-09-28T14:33:59.344498Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['flooding' 'impersonation' 'injection' 'normal']\n",
      "[0 1 2 3]\n"
     ]
    }
   ],
   "source": [
    "print encoder.classes_\n",
    "print binarizer.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T18:52:45.687344Z",
     "start_time": "2018-09-28T18:52:45.679341Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T18:53:07.999606Z",
     "start_time": "2018-09-28T18:53:07.988487Z"
    }
   },
   "outputs": [],
   "source": [
    "# Baseline model for the neural network. We choose a hidden layer of 10 neurons. The lesser number of neurons helps to eliminate the redundancies in the data and select the more important features.\n",
    "def create_baseline_model(n, input_dim):\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(n, input_dim=input_dim, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(4, kernel_initializer='normal', activation='sigmoid'))\n",
    "    # Compile model. We use the the logarithmic loss function, and the Adam gradient optimizer.\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T18:53:08.496424Z",
     "start_time": "2018-09-28T18:53:08.486055Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.wrappers.scikit_learn.KerasClassifier at 0x149c1c210>"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "KerasClassifier(build_fn=create_baseline_model, epochs=100, batch_size=5, verbose=0, n=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T18:53:09.112858Z",
     "start_time": "2018-09-28T18:53:09.104215Z"
    }
   },
   "outputs": [],
   "source": [
    "# use the KerasClassifier\n",
    "\n",
    "preprocessing = Pipeline([\n",
    "    (\"scale\", StandardScaler()),\n",
    "])\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    (\"preprocessing\", preprocessing),\n",
    "    (\"classifier\", KerasClassifier(build_fn=create_baseline_model, epochs=2, batch_size=128, \n",
    "                                   verbose=1, n=10, input_dim=74))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T18:58:10.066000Z",
     "start_time": "2018-09-28T18:55:40.028197Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "892937/892937 [==============================] - 21s 24us/step - loss: 0.1027 - acc: 0.9683\n",
      "Epoch 2/2\n",
      "892937/892937 [==============================] - 18s 20us/step - loss: 0.0314 - acc: 0.9910\n",
      "446469/446469 [==============================] - 4s 10us/step\n",
      "Epoch 1/2\n",
      "892937/892937 [==============================] - 24s 27us/step - loss: 0.1089 - acc: 0.9682\n",
      "Epoch 2/2\n",
      "892937/892937 [==============================] - 19s 22us/step - loss: 0.0305 - acc: 0.9919 0s - loss: 0.0\n",
      "446469/446469 [==============================] - 4s 9us/step\n",
      "Epoch 1/2\n",
      "892938/892938 [==============================] - 18s 20us/step - loss: 0.0619 - acc: 0.9815\n",
      "Epoch 2/2\n",
      "892938/892938 [==============================] - 17s 20us/step - loss: 0.0153 - acc: 0.9916\n",
      "446468/446468 [==============================] - 4s 9us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.97450887, 0.99176875, 0.74421683])"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(pipeline, X, binarized_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T19:53:18.156371Z",
     "start_time": "2018-09-28T19:53:18.149702Z"
    }
   },
   "outputs": [],
   "source": [
    "# notice the LARGE variance in scores of a neural network. This is due to the high-variance nature of how networks fit\n",
    "# using stochastic gradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T19:54:23.033967Z",
     "start_time": "2018-09-28T19:53:19.943756Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "1339406/1339406 [==============================] - 29s 22us/step - loss: 0.0781 - acc: 0.9740\n",
      "Epoch 2/2\n",
      "1339406/1339406 [==============================] - 25s 19us/step - loss: 0.0298 - acc: 0.9856\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('preprocessing', Pipeline(memory=None,\n",
       "     steps=[('scale', StandardScaler(copy=True, with_mean=True, with_std=True))])), ('classifier', <keras.wrappers.scikit_learn.KerasClassifier object at 0x149c1c350>)])"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(X, binarized_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T19:54:23.109606Z",
     "start_time": "2018-09-28T19:54:23.040186Z"
    }
   },
   "outputs": [],
   "source": [
    "# remake \n",
    "encoded_y_test = encoder.transform(y_test)\n",
    "def get_network_test_accuracy_of(model):\n",
    "    y_preds = model.predict(X_test)\n",
    "    return accuracy_score(y_preds, encoded_y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T19:54:26.646671Z",
     "start_time": "2018-09-28T19:54:23.115614Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "389185/389185 [==============================] - 3s 7us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.889327697624523"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# not the best accuracy\n",
    "get_network_test_accuracy_of(pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T19:55:37.670894Z",
     "start_time": "2018-09-28T19:54:26.652739Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "1339406/1339406 [==============================] - 29s 21us/step - loss: 0.0844 - acc: 0.9735 0s - loss: 0.085\n",
      "Epoch 2/2\n",
      "1339406/1339406 [==============================] - 32s 24us/step - loss: 0.0323 - acc: 0.9853 0s - loss: 0.0323 - acc: 0\n",
      "389185/389185 [==============================] - 4s 11us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8742526048023433"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# by fitting again, we get a different test accuracy. This also highlights the variance on the network\n",
    "pipeline.fit(X, binarized_y)\n",
    "get_network_test_accuracy_of(pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T20:05:12.878999Z",
     "start_time": "2018-09-28T19:55:37.676920Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "892937/892937 [==============================] - 20s 22us/step - loss: 0.0945 - acc: 0.9744\n",
      "Epoch 2/10\n",
      "892937/892937 [==============================] - 17s 19us/step - loss: 0.0349 - acc: 0.9906\n",
      "Epoch 3/10\n",
      "892937/892937 [==============================] - 16s 18us/step - loss: 0.0293 - acc: 0.9920\n",
      "Epoch 4/10\n",
      "892937/892937 [==============================] - 17s 20us/step - loss: 0.0261 - acc: 0.9932\n",
      "Epoch 5/10\n",
      "892937/892937 [==============================] - 18s 20us/step - loss: 0.0231 - acc: 0.9938 0s - loss: 0.0232 - ac\n",
      "Epoch 6/10\n",
      "892937/892937 [==============================] - 15s 17us/step - loss: 0.0216 - acc: 0.9941\n",
      "Epoch 7/10\n",
      "892937/892937 [==============================] - 21s 23us/step - loss: 0.0206 - acc: 0.9944\n",
      "Epoch 8/10\n",
      "892937/892937 [==============================] - 17s 20us/step - loss: 0.0199 - acc: 0.9947 0s - loss: 0.0198 - a\n",
      "Epoch 9/10\n",
      "892937/892937 [==============================] - 17s 19us/step - loss: 0.0194 - acc: 0.9948\n",
      "Epoch 10/10\n",
      "892937/892937 [==============================] - 17s 19us/step - loss: 0.0189 - acc: 0.9950\n",
      "446469/446469 [==============================] - 4s 10us/step\n",
      "Epoch 1/10\n",
      "892937/892937 [==============================] - 19s 21us/step - loss: 0.1160 - acc: 0.9618\n",
      "Epoch 2/10\n",
      "892937/892937 [==============================] - 17s 19us/step - loss: 0.0427 - acc: 0.9843\n",
      "Epoch 3/10\n",
      "892937/892937 [==============================] - 18s 20us/step - loss: 0.0347 - acc: 0.9875\n",
      "Epoch 4/10\n",
      "892937/892937 [==============================] - 18s 20us/step - loss: 0.0318 - acc: 0.9895\n",
      "Epoch 5/10\n",
      "892937/892937 [==============================] - 17s 19us/step - loss: 0.0248 - acc: 0.9937\n",
      "Epoch 6/10\n",
      "892937/892937 [==============================] - 18s 20us/step - loss: 0.0225 - acc: 0.9942\n",
      "Epoch 7/10\n",
      "892937/892937 [==============================] - 18s 20us/step - loss: 0.0205 - acc: 0.9948\n",
      "Epoch 8/10\n",
      "892937/892937 [==============================] - 17s 20us/step - loss: 0.0193 - acc: 0.9951\n",
      "Epoch 9/10\n",
      "892937/892937 [==============================] - 17s 20us/step - loss: 0.0182 - acc: 0.9954\n",
      "Epoch 10/10\n",
      "892937/892937 [==============================] - 18s 20us/step - loss: 0.0173 - acc: 0.9956\n",
      "446469/446469 [==============================] - 4s 8us/step\n",
      "Epoch 1/10\n",
      "892938/892938 [==============================] - 26s 29us/step - loss: 0.0552 - acc: 0.9865\n",
      "Epoch 2/10\n",
      "892938/892938 [==============================] - 23s 25us/step - loss: 0.0068 - acc: 0.9959\n",
      "Epoch 3/10\n",
      "892938/892938 [==============================] - 19s 21us/step - loss: 0.0054 - acc: 0.9961\n",
      "Epoch 4/10\n",
      "892938/892938 [==============================] - 19s 21us/step - loss: 0.0048 - acc: 0.9962\n",
      "Epoch 5/10\n",
      "892938/892938 [==============================] - 18s 20us/step - loss: 0.0043 - acc: 0.9964\n",
      "Epoch 6/10\n",
      "892938/892938 [==============================] - 18s 20us/step - loss: 0.0040 - acc: 0.9965\n",
      "Epoch 7/10\n",
      "892938/892938 [==============================] - 18s 20us/step - loss: 0.0038 - acc: 0.9965\n",
      "Epoch 8/10\n",
      "892938/892938 [==============================] - 18s 20us/step - loss: 0.0037 - acc: 0.9965\n",
      "Epoch 9/10\n",
      "892938/892938 [==============================] - 17s 19us/step - loss: 0.0035 - acc: 0.9966\n",
      "Epoch 10/10\n",
      "892938/892938 [==============================] - 18s 20us/step - loss: 0.0035 - acc: 0.9966\n",
      "446468/446468 [==============================] - 4s 8us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.97399595, 0.9939951 , 0.74381591])"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add some more epochs to learn more\n",
    "preprocessing = Pipeline([\n",
    "    (\"scale\", StandardScaler()),\n",
    "])\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    (\"preprocessing\", preprocessing),\n",
    "    (\"classifier\", KerasClassifier(build_fn=create_baseline_model, epochs=10, batch_size=128, \n",
    "                                   verbose=1, n=10, input_dim=74))\n",
    "])\n",
    "\n",
    "cross_val_score(pipeline, X, binarized_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T20:10:22.910329Z",
     "start_time": "2018-09-28T20:05:12.885980Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1339406/1339406 [==============================] - 30s 22us/step - loss: 0.0812 - acc: 0.9754\n",
      "Epoch 2/10\n",
      "1339406/1339406 [==============================] - 27s 20us/step - loss: 0.0280 - acc: 0.9915\n",
      "Epoch 3/10\n",
      "1339406/1339406 [==============================] - 28s 21us/step - loss: 0.0226 - acc: 0.9921\n",
      "Epoch 4/10\n",
      "1339406/1339406 [==============================] - 27s 20us/step - loss: 0.0193 - acc: 0.9940\n",
      "Epoch 5/10\n",
      "1339406/1339406 [==============================] - 28s 21us/step - loss: 0.0169 - acc: 0.9951\n",
      "Epoch 6/10\n",
      "1339406/1339406 [==============================] - 34s 25us/step - loss: 0.0155 - acc: 0.9955\n",
      "Epoch 7/10\n",
      "1339406/1339406 [==============================] - 38s 28us/step - loss: 0.0148 - acc: 0.9957\n",
      "Epoch 8/10\n",
      "1339406/1339406 [==============================] - 34s 25us/step - loss: 0.0143 - acc: 0.9958 3s -\n",
      "Epoch 9/10\n",
      "1339406/1339406 [==============================] - 29s 21us/step - loss: 0.0139 - acc: 0.9960\n",
      "Epoch 10/10\n",
      "1339406/1339406 [==============================] - 28s 21us/step - loss: 0.0134 - acc: 0.9961\n",
      "389185/389185 [==============================] - 3s 8us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8725027943009109"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# by fitting again, we get a different test accuracy. This also highlights the variance on the network\n",
    "pipeline.fit(X, binarized_y)\n",
    "get_network_test_accuracy_of(pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T20:10:22.921878Z",
     "start_time": "2018-09-28T20:10:22.915472Z"
    }
   },
   "outputs": [],
   "source": [
    "# Took much longer.... and still didn't increase accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T20:10:22.943613Z",
     "start_time": "2018-09-28T20:10:22.927468Z"
    }
   },
   "outputs": [],
   "source": [
    "# change our function to have multiple hidden layers in our network\n",
    "def network_builder(hidden_dimensions, input_dim):\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(hidden_dimensions[0], input_dim=input_dim, kernel_initializer='normal', activation='relu'))\n",
    "    # add multiple hidden layers\n",
    "    for dimension in hidden_dimensions[1:]:\n",
    "        model.add(Dense(dimension, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(4, kernel_initializer='normal', activation='sigmoid'))\n",
    "    # Compile model. We use the the logarithmic loss function, and the Adam gradient optimizer.\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T20:22:26.785765Z",
     "start_time": "2018-09-28T20:10:22.949286Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "892937/892937 [==============================] - 24s 26us/step - loss: 0.0457 - acc: 0.9860\n",
      "Epoch 2/10\n",
      "892937/892937 [==============================] - 21s 24us/step - loss: 0.0113 - acc: 0.9967\n",
      "Epoch 3/10\n",
      "892937/892937 [==============================] - 21s 23us/step - loss: 0.0079 - acc: 0.9977\n",
      "Epoch 4/10\n",
      "892937/892937 [==============================] - 26s 29us/step - loss: 0.0066 - acc: 0.9982\n",
      "Epoch 5/10\n",
      "892937/892937 [==============================] - 24s 27us/step - loss: 0.0061 - acc: 0.9983\n",
      "Epoch 6/10\n",
      "892937/892937 [==============================] - 25s 28us/step - loss: 0.0057 - acc: 0.9984 \n",
      "Epoch 7/10\n",
      "892937/892937 [==============================] - 24s 27us/step - loss: 0.0051 - acc: 0.9985\n",
      "Epoch 8/10\n",
      "892937/892937 [==============================] - 24s 27us/step - loss: 0.0050 - acc: 0.9986\n",
      "Epoch 9/10\n",
      "892937/892937 [==============================] - 25s 28us/step - loss: 0.0046 - acc: 0.9986\n",
      "Epoch 10/10\n",
      "892937/892937 [==============================] - 23s 26us/step - loss: 0.0044 - acc: 0.9987\n",
      "446469/446469 [==============================] - 6s 12us/step\n",
      "Epoch 1/10\n",
      "892937/892937 [==============================] - 27s 30us/step - loss: 0.0538 - acc: 0.9826\n",
      "Epoch 2/10\n",
      "892937/892937 [==============================] - 24s 27us/step - loss: 0.0113 - acc: 0.9967\n",
      "Epoch 3/10\n",
      "892937/892937 [==============================] - 21s 23us/step - loss: 0.0081 - acc: 0.9976\n",
      "Epoch 4/10\n",
      "892937/892937 [==============================] - 22s 25us/step - loss: 0.0066 - acc: 0.9981\n",
      "Epoch 5/10\n",
      "892937/892937 [==============================] - 22s 24us/step - loss: 0.0057 - acc: 0.9984\n",
      "Epoch 6/10\n",
      "892937/892937 [==============================] - 27s 30us/step - loss: 0.0052 - acc: 0.9985\n",
      "Epoch 7/10\n",
      "892937/892937 [==============================] - 23s 26us/step - loss: 0.0047 - acc: 0.9987\n",
      "Epoch 8/10\n",
      "892937/892937 [==============================] - 21s 24us/step - loss: 0.0045 - acc: 0.9987  - ETA: 1s\n",
      "Epoch 9/10\n",
      "892937/892937 [==============================] - 22s 25us/step - loss: 0.0043 - acc: 0.9987\n",
      "Epoch 10/10\n",
      "892937/892937 [==============================] - 22s 24us/step - loss: 0.0041 - acc: 0.9988\n",
      "446469/446469 [==============================] - 7s 15us/step\n",
      "Epoch 1/10\n",
      "892938/892938 [==============================] - 28s 31us/step - loss: 0.0416 - acc: 0.9909\n",
      "Epoch 2/10\n",
      "892938/892938 [==============================] - 24s 26us/step - loss: 0.0029 - acc: 0.9992\n",
      "Epoch 3/10\n",
      "892938/892938 [==============================] - 29s 32us/step - loss: 0.0024 - acc: 0.9993\n",
      "Epoch 4/10\n",
      "892938/892938 [==============================] - 22s 24us/step - loss: 0.0021 - acc: 0.9994\n",
      "Epoch 5/10\n",
      "892938/892938 [==============================] - 19s 22us/step - loss: 0.0018 - acc: 0.9995\n",
      "Epoch 6/10\n",
      "892938/892938 [==============================] - 19s 21us/step - loss: 0.0015 - acc: 0.9995\n",
      "Epoch 7/10\n",
      "892938/892938 [==============================] - 22s 24us/step - loss: 0.0016 - acc: 0.9995\n",
      "Epoch 8/10\n",
      "892938/892938 [==============================] - 20s 22us/step - loss: 0.0014 - acc: 0.9996\n",
      "Epoch 9/10\n",
      "892938/892938 [==============================] - 19s 22us/step - loss: 0.0013 - acc: 0.9996\n",
      "Epoch 10/10\n",
      "892938/892938 [==============================] - 19s 21us/step - loss: 0.0013 - acc: 0.9996\n",
      "446468/446468 [==============================] - 4s 10us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.97451335, 0.99801778, 0.73468647])"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add some more hidden layers to learn more\n",
    "preprocessing = Pipeline([\n",
    "    (\"scale\", StandardScaler()),\n",
    "])\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    (\"preprocessing\", preprocessing),\n",
    "    (\"classifier\", KerasClassifier(build_fn=network_builder, epochs=10, batch_size=128, \n",
    "                                   verbose=1, hidden_dimensions=(60,30,10), input_dim=74))\n",
    "])\n",
    "\n",
    "cross_val_score(pipeline, X, binarized_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T20:27:25.004357Z",
     "start_time": "2018-09-28T20:22:26.793376Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1339406/1339406 [==============================] - 31s 23us/step - loss: 0.0422 - acc: 0.9865\n",
      "Epoch 2/10\n",
      "1339406/1339406 [==============================] - 28s 21us/step - loss: 0.0095 - acc: 0.9973\n",
      "Epoch 3/10\n",
      "1339406/1339406 [==============================] - 29s 22us/step - loss: 0.0068 - acc: 0.9981\n",
      "Epoch 4/10\n",
      "1339406/1339406 [==============================] - 28s 21us/step - loss: 0.0056 - acc: 0.9984\n",
      "Epoch 5/10\n",
      "1339406/1339406 [==============================] - 29s 21us/step - loss: 0.0051 - acc: 0.9986\n",
      "Epoch 6/10\n",
      "1339406/1339406 [==============================] - 28s 21us/step - loss: 0.0047 - acc: 0.9987\n",
      "Epoch 7/10\n",
      "1339406/1339406 [==============================] - 30s 22us/step - loss: 0.0041 - acc: 0.9988 0s - loss: 0.0041 - acc: 0.99 - ETA: 0s - loss: 0.0041 - acc: 0.998 - ETA: 0s - loss: 0.0041 - acc: 0\n",
      "Epoch 8/10\n",
      "1339406/1339406 [==============================] - 29s 22us/step - loss: 0.0039 - acc: 0.9989\n",
      "Epoch 9/10\n",
      "1339406/1339406 [==============================] - 29s 22us/step - loss: 0.0039 - acc: 0.9989\n",
      "Epoch 10/10\n",
      "1339406/1339406 [==============================] - 28s 21us/step - loss: 0.0036 - acc: 0.9990 0s - loss: 0.0036 - acc: \n",
      "389185/389185 [==============================] - 3s 9us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8897876331307732"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(X, binarized_y)\n",
    "get_network_test_accuracy_of(pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T20:27:25.020513Z",
     "start_time": "2018-09-28T20:27:25.014589Z"
    }
   },
   "outputs": [],
   "source": [
    "# we got small bump by increasing hidden layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T20:49:49.754149Z",
     "start_time": "2018-09-28T20:38:14.375087Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "892937/892937 [==============================] - 25s 28us/step - loss: 0.0671 - acc: 0.9709\n",
      "Epoch 2/10\n",
      "892937/892937 [==============================] - 21s 23us/step - loss: 0.0139 - acc: 0.9963\n",
      "Epoch 3/10\n",
      "892937/892937 [==============================] - 20s 22us/step - loss: 0.0100 - acc: 0.9973\n",
      "Epoch 4/10\n",
      "892937/892937 [==============================] - 25s 28us/step - loss: 0.0087 - acc: 0.9977\n",
      "Epoch 5/10\n",
      "892937/892937 [==============================] - 21s 24us/step - loss: 0.0078 - acc: 0.9979\n",
      "Epoch 6/10\n",
      "892937/892937 [==============================] - 21s 24us/step - loss: 0.0072 - acc: 0.9981\n",
      "Epoch 7/10\n",
      "892937/892937 [==============================] - 24s 27us/step - loss: 0.0069 - acc: 0.9982\n",
      "Epoch 8/10\n",
      "892937/892937 [==============================] - 24s 27us/step - loss: 0.0064 - acc: 0.9984\n",
      "Epoch 9/10\n",
      "892937/892937 [==============================] - 25s 28us/step - loss: 0.0060 - acc: 0.9984\n",
      "Epoch 10/10\n",
      "892937/892937 [==============================] - 22s 25us/step - loss: 0.0059 - acc: 0.9985\n",
      "446469/446469 [==============================] - 5s 11us/step\n",
      "Epoch 1/10\n",
      "892937/892937 [==============================] - 26s 29us/step - loss: 0.1102 - acc: 0.9298\n",
      "Epoch 2/10\n",
      "892937/892937 [==============================] - 24s 27us/step - loss: 0.0549 - acc: 0.9447\n",
      "Epoch 3/10\n",
      "892937/892937 [==============================] - 22s 24us/step - loss: 0.0456 - acc: 0.9494\n",
      "Epoch 4/10\n",
      "892937/892937 [==============================] - 22s 24us/step - loss: 0.0354 - acc: 0.9606\n",
      "Epoch 5/10\n",
      "892937/892937 [==============================] - 21s 23us/step - loss: 0.0342 - acc: 0.9609\n",
      "Epoch 6/10\n",
      "892937/892937 [==============================] - 20s 23us/step - loss: 0.0333 - acc: 0.9611\n",
      "Epoch 7/10\n",
      "892937/892937 [==============================] - 21s 23us/step - loss: 0.0329 - acc: 0.9613 1s\n",
      "Epoch 8/10\n",
      "892937/892937 [==============================] - 20s 23us/step - loss: 0.0256 - acc: 0.9720\n",
      "Epoch 9/10\n",
      "892937/892937 [==============================] - 20s 23us/step - loss: 0.0061 - acc: 0.9984 1s - loss: 0.0062 - acc: - ETA: 0s - loss: 0.006\n",
      "Epoch 10/10\n",
      "892937/892937 [==============================] - 21s 24us/step - loss: 0.0058 - acc: 0.9984 0s - loss: 0.0058 - acc: 0.998\n",
      "446469/446469 [==============================] - 5s 12us/step\n",
      "Epoch 1/10\n",
      "892938/892938 [==============================] - 26s 29us/step - loss: 0.0255 - acc: 0.9952\n",
      "Epoch 2/10\n",
      "892938/892938 [==============================] - 24s 27us/step - loss: 0.0031 - acc: 0.9992\n",
      "Epoch 3/10\n",
      "892938/892938 [==============================] - 20s 23us/step - loss: 0.0025 - acc: 0.9994\n",
      "Epoch 4/10\n",
      "892938/892938 [==============================] - 21s 23us/step - loss: 0.0022 - acc: 0.9995\n",
      "Epoch 5/10\n",
      "892938/892938 [==============================] - 21s 24us/step - loss: 0.0019 - acc: 0.9995 1s - loss:\n",
      "Epoch 6/10\n",
      "892938/892938 [==============================] - 21s 24us/step - loss: 0.0019 - acc: 0.9995 1s - loss\n",
      "Epoch 7/10\n",
      "892938/892938 [==============================] - 21s 24us/step - loss: 0.0017 - acc: 0.9995\n",
      "Epoch 8/10\n",
      "892938/892938 [==============================] - 21s 23us/step - loss: 0.0016 - acc: 0.9995\n",
      "Epoch 9/10\n",
      "892938/892938 [==============================] - 20s 23us/step - loss: 0.0014 - acc: 0.9996\n",
      "Epoch 10/10\n",
      "892938/892938 [==============================] - 21s 24us/step - loss: 0.0013 - acc: 0.9996\n",
      "446468/446468 [==============================] - 5s 12us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.97447527, 0.99417877, 0.74292446])"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add some more hidden layers to learn more\n",
    "preprocessing = Pipeline([\n",
    "    (\"scale\", StandardScaler()),\n",
    "])\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    (\"preprocessing\", preprocessing),\n",
    "    (\"classifier\", KerasClassifier(build_fn=network_builder, epochs=10, batch_size=128, \n",
    "                                   verbose=1, hidden_dimensions=(30,30,30,10), input_dim=74))\n",
    "])\n",
    "\n",
    "cross_val_score(pipeline, X, binarized_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T21:14:14.753541Z",
     "start_time": "2018-09-28T20:59:05.927746Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1339406/1339406 [==============================] - 48s 36us/step - loss: 0.0666 - acc: 0.9548\n",
      "Epoch 2/10\n",
      "1339406/1339406 [==============================] - 108s 81us/step - loss: 0.0346 - acc: 0.9663\n",
      "Epoch 3/10\n",
      "1339406/1339406 [==============================] - 78s 59us/step - loss: 0.0261 - acc: 0.9732\n",
      "Epoch 4/10\n",
      "1339406/1339406 [==============================] - 102s 76us/step - loss: 0.0075 - acc: 0.9980\n",
      "Epoch 5/10\n",
      "1339406/1339406 [==============================] - 71s 53us/step - loss: 0.0066 - acc: 0.9983\n",
      "Epoch 6/10\n",
      "1339406/1339406 [==============================] - 111s 83us/step - loss: 0.0059 - acc: 0.9985\n",
      "Epoch 7/10\n",
      "1339406/1339406 [==============================] - 98s 73us/step - loss: 0.0055 - acc: 0.9986\n",
      "Epoch 8/10\n",
      "1339406/1339406 [==============================] - 93s 70us/step - loss: 0.0052 - acc: 0.9987\n",
      "Epoch 9/10\n",
      "1339406/1339406 [==============================] - 88s 66us/step - loss: 0.0051 - acc: 0.9988\n",
      "Epoch 10/10\n",
      "1339406/1339406 [==============================] - 87s 65us/step - loss: 0.0049 - acc: 0.9988\n",
      "389185/389185 [==============================] - 16s 41us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8899315235684828"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(X, binarized_y)\n",
    "get_network_test_accuracy_of(pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T21:27:16.043318Z",
     "start_time": "2018-09-28T21:27:16.036475Z"
    }
   },
   "outputs": [],
   "source": [
    "# best so far using deep learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# deep learning isn't best for all data sets."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
